"""
@author: yqx
@file: date_gain.py
@time: 2024/6/18 19:24
"""
import time
import requests
from bs4 import BeautifulSoup

def download(url):
    # 告诉目标站点字节信息
    headers = {
        # "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0"
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0"
    }
    try:
        # 内部写有可能会出错的代码
        response=requests.get(url,headers=headers)
        response.raise_for_status()
        # 设置编码格式
        # response.encoding=response.apparent_encoding
        response.encoding="UTF-8"
        # 返回文本数据
        time.sleep(1.5)
        print(url,"访问成功")
        return response.text
    except Exception as ex:
        # 如果出错了会执行这里
        print(url,"访问错误:", ex)
        return ""

urls="https://www.zhipin.com/"
text=download(urls)
    # 如果请求失败，则抛出HTTPError异常
    # response.raise_for_status()
soup=BeautifulSoup(text,'lxml')
# print(soup)

job_type=soup.find_all(class_="menu-sub")
# print(job_type)
import csv
with open("position-data.csv",'w',encoding='utf-8')as wr:
    csvwriter=csv.writer(wr)
    headers=["行业","方向","职位"]
    csvwriter.writerow(headers)
    for industry in job_type:
        # print(industry)
        ct1=industry.find_all('li')
        # print(ts1)
        indu = industry.find('p').get_text(strip=True)
        # indu=indu.get_text(strip=True)
        # print(indu)
        for direction in ct1:
            # print(direction)
            dire=direction.find('h4').get_text(strip=True)
            # dire=dire.get_text(strip=True)
            # print(dire)
            ct2=direction.find_all('a')
            # print(pro)
            for profession in ct2:
                prof=profession.get_text(strip=True)
                # print(prof)
                row_data=[indu,dire,prof]
                csvwriter.writerow(row_data)

import pandas as pd
fpath="./position-data.csv"
pd=pd.read_csv(fpath)
print(pd)

# https://www.zhipin.com/web/geek/job?city=101010100&position=100901